import sys

from langchain.text_splitter import RecursiveCharacterTextSplitter
from dalchemy.data import TextHelper


def main():
    helper = TextHelper()
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=100)
    infile = sys.argv[1]
    outfile = sys.argv[2]

    text = helper.read_file(infile)
    texts = text_splitter.split_text(text)
    results = []
    for pid, text in enumerate(texts):
        text = text.strip()
        if text == "": continue
        data = {"pid": pid, "len": len(text), "content": text}
        results.append(data)
    helper.write_json(results, outfile)


if __name__ == '__main__':
    main()
